#!/usr/bin/env python 
# -*- coding:utf-8 -*-
import time

from app.db.mongo_db import Mongo_db
from app.douyin import mongo_db_host, mongo_db_port
from app.douyin.proxy_test import proxy_test


def get_uid_db(page_num=0, page_count=50):
    """
    查询数据库
    :param page_num: 页码
    :param page_count: 记录数
    :return:
    """
    uid_list = []
    # 查询数据库
    douyin_db = Mongo_db(mongo_db_host, mongo_db_port).get_db('douyin')
    user_coll = douyin_db['user']
    user_list = user_coll.find({"uid": {"$exists": True}}).skip((page_num - 1) * page_count).limit(page_count)
    if user_list:
        for index, user in enumerate(user_list):
            uid = user.get('uid')
            if uid:
                # print("数据库中uid存在 uid = {}".format(uid))
                uid_list.append(uid)
            else:
                print("数据库中uid不存在")

    return uid_list


def spider_main():
    """
    抖音分享页爬虫入口
    :return:
    """
    # 查询已知用户数目
    page = 1
    # 初始化配置
    dy = proxy_test(type='chrome')
    # dy = proxy_test(type='firefox')
    # 初始化记录数
    uid_list = get_uid_db(page)

    # 读取文件获取uid
    # with open('./uid_list.txt', 'r', encoding='utf-8') as f:
    #     for line in f:
    #         line = line.replace('\n', '').replace(' ', '')
    #         if line:
    #             uid_list.append(line)

    # 根据uid开始爬取数据
    while len(uid_list) > 0:
        print("page = %d" % page)
        # 开始遍历执行
        for index, uid in enumerate(uid_list):
            print("index={}, uid={}".format(index, uid))
            if uid:
                url = 'https://www.iesdouyin.com/share/user/{}'.format(uid)
                print("uid 存在，生成url=%s" % url)

                # 爬虫main
                dy.main(url)
            else:
                print("uid 不存在")

            print("休息，准备查看下个记录")
            time.sleep(1)

        page += 1
        uid_list = get_uid_db(page)

        print("休息一下，等待下一页数据")
        time.sleep(5 * 60)

    # dy.quit()  # 退出程序，但是进程还是保留了。。。


if __name__ == '__main__':
    print("test 1111")
    spider_main()
